<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 5.2.0">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/logo.svg" color="#222">

<link rel="stylesheet" href="/css/main.css">

<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic&display=swap&subset=latin,latin-ext">
<link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>

<script id="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"blog.hory-ai.com","root":"/","scheme":"Pisces","version":"7.8.0","exturl":false,"sidebar":{"position":"right","Muse | Mist":320,"display":"always","padding":18,"offset":12,"onmobile":false},"copycode":{"enable":true,"show_result":true,"style":"mac"},"back2top":{"enable":true,"sidebar":false,"scrollpercent":true},"bookmark":{"enable":true,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"algolia":{"hits":{"per_page":10},"labels":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}},"localsearch":{"enable":true,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},"path":"search.xml"};
  </script>

  <meta name="description" content="获取scrapy request 头部信息 及捕捉异常信息 1234567891011121314151617181920212223242526# -*- coding: utf-8 -*-import scrapyfrom Amazon.items import AmazonItemfrom scrapy_redis_sentinel.spiders import RedisSpiderfro">
<meta property="og:type" content="article">
<meta property="og:title" content="获取scrapy request 头部信息及捕捉异常信息">
<meta property="og:url" content="http://blog.hory-ai.com/2020/07/29/%E8%8E%B7%E5%8F%96scrapy-request-%E5%A4%B4%E9%83%A8%E4%BF%A1%E6%81%AF%E5%8F%8A%E6%8D%95%E6%8D%89%E5%BC%82%E5%B8%B8%E4%BF%A1%E6%81%AF/index.html">
<meta property="og:site_name" content="Horysk 宏睿时空">
<meta property="og:description" content="获取scrapy request 头部信息 及捕捉异常信息 1234567891011121314151617181920212223242526# -*- coding: utf-8 -*-import scrapyfrom Amazon.items import AmazonItemfrom scrapy_redis_sentinel.spiders import RedisSpiderfro">
<meta property="og:locale" content="en_US">
<meta property="article:published_time" content="2020-07-29T17:35:56.000Z">
<meta property="article:modified_time" content="2020-07-29T17:35:56.000Z">
<meta property="article:author" content="Hory Skone">
<meta name="twitter:card" content="summary">

<link rel="canonical" href="http://blog.hory-ai.com/2020/07/29/%E8%8E%B7%E5%8F%96scrapy-request-%E5%A4%B4%E9%83%A8%E4%BF%A1%E6%81%AF%E5%8F%8A%E6%8D%95%E6%8D%89%E5%BC%82%E5%B8%B8%E4%BF%A1%E6%81%AF/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'en'
  };
</script>

  <title>获取scrapy request 头部信息及捕捉异常信息 | Horysk 宏睿时空</title>
  






  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container use-motion">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="Toggle navigation bar">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <h1 class="site-title">Horysk 宏睿时空</h1>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
        <i class="fa fa-search fa-fw fa-lg"></i>
    </div>
  </div>
</div>




<nav class="site-nav">
  <ul id="menu" class="main-menu menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="fa fa-home fa-fw"></i>Home</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="fa fa-tags fa-fw"></i>Tags</a>

  </li>
        <li class="menu-item menu-item-categories">

    <a href="/categories/" rel="section"><i class="fa fa-th fa-fw"></i>Categories</a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="fa fa-archive fa-fw"></i>Archives</a>

  </li>
      <li class="menu-item menu-item-search">
        <a role="button" class="popup-trigger"><i class="fa fa-search fa-fw"></i>Search
        </a>
      </li>
  </ul>
</nav>



  <div class="search-pop-overlay">
    <div class="popup search-popup">
        <div class="search-header">
  <span class="search-icon">
    <i class="fa fa-search"></i>
  </span>
  <div class="search-input-container">
    <input autocomplete="off" autocapitalize="off"
           placeholder="Searching..." spellcheck="false"
           type="search" class="search-input">
  </div>
  <span class="popup-btn-close">
    <i class="fa fa-times-circle"></i>
  </span>
</div>
<div id="search-result">
  <div id="no-result">
    <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
  </div>
</div>

    </div>
  </div>

</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>
  <div class="reading-progress-bar"></div>
  <a role="button" class="book-mark-link book-mark-link-fixed"></a>

  <a href="https://github.com/horysk" class="github-corner" title="Follow me on GitHub" aria-label="Follow me on GitHub" rel="noopener" target="_blank"><svg width="80" height="80" viewBox="0 0 250 250" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content post posts-expand">
            

    
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="http://blog.hory-ai.com/2020/07/29/%E8%8E%B7%E5%8F%96scrapy-request-%E5%A4%B4%E9%83%A8%E4%BF%A1%E6%81%AF%E5%8F%8A%E6%8D%95%E6%8D%89%E5%BC%82%E5%B8%B8%E4%BF%A1%E6%81%AF/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Hory Skone">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Horysk 宏睿时空">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          获取scrapy request 头部信息及捕捉异常信息
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2020-07-29 17:35:56" itemprop="dateCreated datePublished" datetime="2020-07-29T17:35:56+00:00">2020-07-29</time>
            </span>

          
            <span class="post-meta-item" title="Views" id="busuanzi_container_page_pv" style="display: none;">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">Views: </span>
              <span id="busuanzi_value_page_pv"></span>
            </span>
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="far fa-comment"></i>
      </span>
      <span class="post-meta-item-text">Valine: </span>
    
    <a title="valine" href="/2020/07/29/%E8%8E%B7%E5%8F%96scrapy-request-%E5%A4%B4%E9%83%A8%E4%BF%A1%E6%81%AF%E5%8F%8A%E6%8D%95%E6%8D%89%E5%BC%82%E5%B8%B8%E4%BF%A1%E6%81%AF/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2020/07/29/%E8%8E%B7%E5%8F%96scrapy-request-%E5%A4%B4%E9%83%A8%E4%BF%A1%E6%81%AF%E5%8F%8A%E6%8D%95%E6%8D%89%E5%BC%82%E5%B8%B8%E4%BF%A1%E6%81%AF/" itemprop="commentCount"></span>
    </a>
  </span>
  
  <br>
            <span class="post-meta-item" title="Symbols count in article">
              <span class="post-meta-item-icon">
                <i class="far fa-file-word"></i>
              </span>
                <span class="post-meta-item-text">Symbols count in article: </span>
              <span>22k</span>
            </span>
            <span class="post-meta-item" title="Reading time">
              <span class="post-meta-item-icon">
                <i class="far fa-clock"></i>
              </span>
                <span class="post-meta-item-text">Reading time &asymp;</span>
              <span>20 mins.</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <p>获取scrapy request 头部信息 及捕捉异常信息</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="keyword">import</span> scrapy</span><br><span class="line"><span class="keyword">from</span> Amazon.items <span class="keyword">import</span> AmazonItem</span><br><span class="line"><span class="keyword">from</span> scrapy_redis_sentinel.spiders <span class="keyword">import</span> RedisSpider</span><br><span class="line"><span class="keyword">from</span> Amazon.tools <span class="keyword">import</span> GetAmazonUrlCountry , GetLanguage,replace_string, replace_list, formtting_rank,formtting_info,GetAmazonUrlDomain,GetAmazonUrlCID</span><br><span class="line"><span class="keyword">from</span> datetime <span class="keyword">import</span> datetime</span><br><span class="line"><span class="keyword">import</span> urllib3</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> dateutil.parser <span class="keyword">import</span> *</span><br><span class="line"><span class="keyword">from</span> Amazon.ConfigDB <span class="keyword">import</span> RedisDB,MongoDB</span><br><span class="line">mongo_conn=MongoDB(<span class="string">&quot;AmazonDB&quot;</span>,<span class="string">&quot;AmazonAddKW&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">AmazonkwSpider</span>(<span class="params">RedisSpider</span>):</span></span><br><span class="line">    name = <span class="string">&#x27;amazonkw&#x27;</span></span><br><span class="line">    redis_key = <span class="string">&quot;amazonkw&quot;</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># def start_requests(self):</span></span><br><span class="line">    <span class="comment">#     url=&quot;http://httpbin.org/ip&quot;</span></span><br><span class="line">    <span class="comment">#     yield scrapy.Request(url,  callback=self.parse)</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse</span>(<span class="params">self, response</span>):</span></span><br><span class="line">        <span class="comment"># res=response.request.headers</span></span><br><span class="line">        <span class="comment"># print(res)</span></span><br><span class="line">        <span class="comment"># print(response.text)</span></span><br></pre></td></tr></table></figure>
<h2 id="scrapy-middleware中间件"><a href="#scrapy-middleware中间件" class="headerlink" title="scrapy  middleware中间件"></a>scrapy  middleware中间件</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br><span class="line">195</span><br><span class="line">196</span><br><span class="line">197</span><br><span class="line">198</span><br><span class="line">199</span><br><span class="line">200</span><br><span class="line">201</span><br><span class="line">202</span><br><span class="line">203</span><br><span class="line">204</span><br><span class="line">205</span><br><span class="line">206</span><br><span class="line">207</span><br><span class="line">208</span><br><span class="line">209</span><br><span class="line">210</span><br><span class="line">211</span><br><span class="line">212</span><br><span class="line">213</span><br><span class="line">214</span><br><span class="line">215</span><br><span class="line">216</span><br><span class="line">217</span><br><span class="line">218</span><br><span class="line">219</span><br><span class="line">220</span><br><span class="line">221</span><br><span class="line">222</span><br><span class="line">223</span><br><span class="line">224</span><br><span class="line">225</span><br><span class="line">226</span><br><span class="line">227</span><br><span class="line">228</span><br><span class="line">229</span><br><span class="line">230</span><br><span class="line">231</span><br><span class="line">232</span><br><span class="line">233</span><br><span class="line">234</span><br><span class="line">235</span><br><span class="line">236</span><br><span class="line">237</span><br><span class="line">238</span><br><span class="line">239</span><br><span class="line">240</span><br><span class="line">241</span><br><span class="line">242</span><br><span class="line">243</span><br><span class="line">244</span><br><span class="line">245</span><br><span class="line">246</span><br><span class="line">247</span><br><span class="line">248</span><br><span class="line">249</span><br><span class="line">250</span><br><span class="line">251</span><br><span class="line">252</span><br><span class="line">253</span><br><span class="line">254</span><br><span class="line">255</span><br><span class="line">256</span><br><span class="line">257</span><br><span class="line">258</span><br><span class="line">259</span><br><span class="line">260</span><br><span class="line">261</span><br><span class="line">262</span><br><span class="line">263</span><br><span class="line">264</span><br><span class="line">265</span><br><span class="line">266</span><br><span class="line">267</span><br><span class="line">268</span><br><span class="line">269</span><br><span class="line">270</span><br><span class="line">271</span><br><span class="line">272</span><br><span class="line">273</span><br><span class="line">274</span><br><span class="line">275</span><br><span class="line">276</span><br><span class="line">277</span><br><span class="line">278</span><br><span class="line">279</span><br><span class="line">280</span><br><span class="line">281</span><br><span class="line">282</span><br><span class="line">283</span><br><span class="line">284</span><br><span class="line">285</span><br><span class="line">286</span><br><span class="line">287</span><br><span class="line">288</span><br><span class="line">289</span><br><span class="line">290</span><br><span class="line">291</span><br><span class="line">292</span><br><span class="line">293</span><br><span class="line">294</span><br><span class="line">295</span><br><span class="line">296</span><br><span class="line">297</span><br><span class="line">298</span><br><span class="line">299</span><br><span class="line">300</span><br><span class="line">301</span><br><span class="line">302</span><br><span class="line">303</span><br><span class="line">304</span><br><span class="line">305</span><br><span class="line">306</span><br><span class="line">307</span><br><span class="line">308</span><br><span class="line">309</span><br><span class="line">310</span><br><span class="line">311</span><br><span class="line">312</span><br><span class="line">313</span><br><span class="line">314</span><br><span class="line">315</span><br><span class="line">316</span><br><span class="line">317</span><br><span class="line">318</span><br><span class="line">319</span><br><span class="line">320</span><br><span class="line">321</span><br><span class="line">322</span><br><span class="line">323</span><br><span class="line">324</span><br><span class="line">325</span><br><span class="line">326</span><br><span class="line">327</span><br><span class="line">328</span><br><span class="line">329</span><br><span class="line">330</span><br><span class="line">331</span><br><span class="line">332</span><br><span class="line">333</span><br><span class="line">334</span><br><span class="line">335</span><br><span class="line">336</span><br><span class="line">337</span><br><span class="line">338</span><br><span class="line">339</span><br><span class="line">340</span><br><span class="line">341</span><br><span class="line">342</span><br><span class="line">343</span><br><span class="line">344</span><br><span class="line">345</span><br><span class="line">346</span><br><span class="line">347</span><br><span class="line">348</span><br><span class="line">349</span><br><span class="line">350</span><br><span class="line">351</span><br><span class="line">352</span><br><span class="line">353</span><br><span class="line">354</span><br><span class="line">355</span><br><span class="line">356</span><br><span class="line">357</span><br><span class="line">358</span><br><span class="line">359</span><br><span class="line">360</span><br><span class="line">361</span><br><span class="line">362</span><br><span class="line">363</span><br><span class="line">364</span><br><span class="line">365</span><br><span class="line">366</span><br><span class="line">367</span><br><span class="line">368</span><br><span class="line">369</span><br><span class="line">370</span><br><span class="line">371</span><br><span class="line">372</span><br><span class="line">373</span><br><span class="line">374</span><br><span class="line">375</span><br><span class="line">376</span><br><span class="line">377</span><br><span class="line">378</span><br><span class="line">379</span><br><span class="line">380</span><br><span class="line">381</span><br><span class="line">382</span><br><span class="line">383</span><br><span class="line">384</span><br><span class="line">385</span><br><span class="line">386</span><br><span class="line">387</span><br><span class="line">388</span><br><span class="line">389</span><br><span class="line">390</span><br><span class="line">391</span><br><span class="line">392</span><br><span class="line">393</span><br><span class="line">394</span><br><span class="line">395</span><br><span class="line">396</span><br><span class="line">397</span><br><span class="line">398</span><br><span class="line">399</span><br><span class="line">400</span><br><span class="line">401</span><br><span class="line">402</span><br><span class="line">403</span><br><span class="line">404</span><br><span class="line">405</span><br><span class="line">406</span><br><span class="line">407</span><br><span class="line">408</span><br><span class="line">409</span><br><span class="line">410</span><br><span class="line">411</span><br><span class="line">412</span><br><span class="line">413</span><br><span class="line">414</span><br><span class="line">415</span><br><span class="line">416</span><br><span class="line">417</span><br><span class="line">418</span><br><span class="line">419</span><br><span class="line">420</span><br><span class="line">421</span><br><span class="line">422</span><br><span class="line">423</span><br><span class="line">424</span><br><span class="line">425</span><br><span class="line">426</span><br><span class="line">427</span><br><span class="line">428</span><br><span class="line">429</span><br><span class="line">430</span><br><span class="line">431</span><br><span class="line">432</span><br><span class="line">433</span><br><span class="line">434</span><br><span class="line">435</span><br><span class="line">436</span><br><span class="line">437</span><br><span class="line">438</span><br><span class="line">439</span><br><span class="line">440</span><br><span class="line">441</span><br><span class="line">442</span><br><span class="line">443</span><br><span class="line">444</span><br><span class="line">445</span><br><span class="line">446</span><br><span class="line">447</span><br><span class="line">448</span><br><span class="line">449</span><br><span class="line">450</span><br><span class="line">451</span><br><span class="line">452</span><br><span class="line">453</span><br><span class="line">454</span><br><span class="line">455</span><br><span class="line">456</span><br><span class="line">457</span><br><span class="line">458</span><br><span class="line">459</span><br><span class="line">460</span><br><span class="line">461</span><br><span class="line">462</span><br><span class="line">463</span><br><span class="line">464</span><br><span class="line">465</span><br><span class="line">466</span><br><span class="line">467</span><br><span class="line">468</span><br><span class="line">469</span><br><span class="line">470</span><br><span class="line">471</span><br><span class="line">472</span><br><span class="line">473</span><br><span class="line">474</span><br><span class="line">475</span><br><span class="line">476</span><br><span class="line">477</span><br><span class="line">478</span><br><span class="line">479</span><br><span class="line">480</span><br><span class="line">481</span><br><span class="line">482</span><br><span class="line">483</span><br><span class="line">484</span><br><span class="line">485</span><br><span class="line">486</span><br><span class="line">487</span><br><span class="line">488</span><br><span class="line">489</span><br><span class="line">490</span><br><span class="line">491</span><br><span class="line">492</span><br><span class="line">493</span><br><span class="line">494</span><br><span class="line">495</span><br><span class="line">496</span><br><span class="line">497</span><br><span class="line">498</span><br><span class="line">499</span><br><span class="line">500</span><br><span class="line">501</span><br><span class="line">502</span><br><span class="line">503</span><br><span class="line">504</span><br><span class="line">505</span><br><span class="line">506</span><br><span class="line">507</span><br><span class="line">508</span><br><span class="line">509</span><br><span class="line">510</span><br><span class="line">511</span><br><span class="line">512</span><br><span class="line">513</span><br><span class="line">514</span><br><span class="line">515</span><br><span class="line">516</span><br><span class="line">517</span><br><span class="line">518</span><br><span class="line">519</span><br><span class="line">520</span><br><span class="line">521</span><br><span class="line">522</span><br><span class="line">523</span><br><span class="line">524</span><br><span class="line">525</span><br><span class="line">526</span><br><span class="line">527</span><br><span class="line">528</span><br><span class="line">529</span><br><span class="line">530</span><br><span class="line">531</span><br><span class="line">532</span><br><span class="line">533</span><br><span class="line">534</span><br><span class="line">535</span><br><span class="line">536</span><br><span class="line">537</span><br><span class="line">538</span><br><span class="line">539</span><br><span class="line">540</span><br><span class="line">541</span><br><span class="line">542</span><br><span class="line">543</span><br><span class="line">544</span><br><span class="line">545</span><br><span class="line">546</span><br><span class="line">547</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># Define here the models for your spider middleware</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># See documentation in:</span></span><br><span class="line"><span class="comment"># https://docs.scrapy.org/en/latest/topics/spider-middleware.html</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> signals</span><br><span class="line"><span class="keyword">from</span> datetime <span class="keyword">import</span> datetime</span><br><span class="line"><span class="keyword">from</span> dateutil.parser <span class="keyword">import</span> *</span><br><span class="line"><span class="keyword">from</span> Amazon.ConfigDB <span class="keyword">import</span> RedisDB,RedisPool,MongoDB</span><br><span class="line"><span class="keyword">from</span> Amazon.tools <span class="keyword">import</span> GetAmazonUrlCountry , GetLanguage,replace_string, replace_list, formtting_rank,formtting_info,GetAmazonUrlDomain,GetAmazonUrlCID</span><br><span class="line">redis_ua=RedisPool()</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">AmazonSpiderMiddleware</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line">    <span class="comment"># Not all methods need to be defined. If a method is not defined,</span></span><br><span class="line">    <span class="comment"># scrapy acts as if the spider middleware does not modify the</span></span><br><span class="line">    <span class="comment"># passed objects.</span></span><br><span class="line"></span><br><span class="line"><span class="meta">    @classmethod</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span>(<span class="params">cls, crawler</span>):</span></span><br><span class="line">        <span class="comment"># This method is used by Scrapy to create your spiders.</span></span><br><span class="line">        s = cls()</span><br><span class="line">        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)</span><br><span class="line">        <span class="keyword">return</span> s</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_spider_input</span>(<span class="params">self, response, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called for each response that goes through the spider</span></span><br><span class="line">        <span class="comment"># middleware and into the spider.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Should return None or raise an exception.</span></span><br><span class="line">        <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_spider_output</span>(<span class="params">self, response, result, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called with the results returned from the Spider, after</span></span><br><span class="line">        <span class="comment"># it has processed the response.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Must return an iterable of Request, dict or Item objects.</span></span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> result:</span><br><span class="line">            <span class="keyword">yield</span> i</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_spider_exception</span>(<span class="params">self, response, exception, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called when a spider or process_spider_input() method</span></span><br><span class="line">        <span class="comment"># (from other spider middleware) raises an exception.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Should return either None or an iterable of Request, dict</span></span><br><span class="line">        <span class="comment"># or Item objects.</span></span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_start_requests</span>(<span class="params">self, start_requests, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called with the start requests of the spider, and works</span></span><br><span class="line">        <span class="comment"># similarly to the process_spider_output() method, except</span></span><br><span class="line">        <span class="comment"># that it doesn’t have a response associated.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Must return only requests (not items).</span></span><br><span class="line">        <span class="keyword">for</span> r <span class="keyword">in</span> start_requests:</span><br><span class="line">            <span class="keyword">yield</span> r</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">spider_opened</span>(<span class="params">self, spider</span>):</span></span><br><span class="line">        spider.logger.info(<span class="string">&#x27;Spider opened: %s&#x27;</span> % spider.name)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">DownloaderMiddleware</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line">    <span class="comment"># Not all methods need to be defined. If a method is not defined,</span></span><br><span class="line">    <span class="comment"># scrapy acts as if the downloader middleware does not modify the</span></span><br><span class="line">    <span class="comment"># passed objects.</span></span><br><span class="line"></span><br><span class="line"><span class="meta">    @classmethod</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span>(<span class="params">cls, crawler</span>):</span></span><br><span class="line">        <span class="comment"># This method is used by Scrapy to create your spiders.</span></span><br><span class="line">        s = cls()</span><br><span class="line">        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)</span><br><span class="line">        <span class="keyword">return</span> s</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_request</span>(<span class="params">self, request, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called for each request that goes through the downloader</span></span><br><span class="line">        <span class="comment"># middleware.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Must either:</span></span><br><span class="line">        <span class="comment"># - return None: continue processing this request</span></span><br><span class="line">        <span class="comment"># - or return a Response object</span></span><br><span class="line">        <span class="comment"># - or return a Request object</span></span><br><span class="line">        <span class="comment"># - or raise IgnoreRequest: process_exception() methods of</span></span><br><span class="line">        <span class="comment">#   installed downloader middleware will be called</span></span><br><span class="line">        <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_response</span>(<span class="params">self, request, response, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called with the response returned from the downloader.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Must either;</span></span><br><span class="line">        <span class="comment"># - return a Response object</span></span><br><span class="line">        <span class="comment"># - return a Request object</span></span><br><span class="line">        <span class="comment"># - or raise IgnoreRequest</span></span><br><span class="line">        <span class="keyword">return</span> response</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_exception</span>(<span class="params">self, request, exception, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called when a download handler or a process_request()</span></span><br><span class="line">        <span class="comment"># (from other downloader middleware) raises an exception.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Must either:</span></span><br><span class="line">        <span class="comment"># - return None: continue processing this exception</span></span><br><span class="line">        <span class="comment"># - return a Response object: stops process_exception() chain</span></span><br><span class="line">        <span class="comment"># - return a Request object: stops process_exception() chain</span></span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">spider_opened</span>(<span class="params">self, spider</span>):</span></span><br><span class="line">        spider.logger.info(<span class="string">&#x27;Spider opened: %s&#x27;</span> % spider.name)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> base64</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="comment"># from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware</span></span><br><span class="line"><span class="keyword">from</span> Amazon.settings <span class="keyword">import</span> USER_AGENTS</span><br><span class="line"><span class="comment"># from settings import PROXIES</span></span><br><span class="line"><span class="comment"># class RandomUserAgent(UserAgentMiddleware):</span></span><br><span class="line"><span class="comment">#     def process_request(self, request, spider):</span></span><br><span class="line"><span class="comment">#         _ua = random.choice(USER_AGENTS)</span></span><br><span class="line"><span class="comment">#         request.headers.setdefault(&#x27;User-Agent&#x27;, _ua)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># class ProxyMiddleware(object):</span></span><br><span class="line"><span class="comment"># 	def process_request(self, request, spider):</span></span><br><span class="line"><span class="comment">#         proxy = requests.get(&quot;http://172.20.0.252:5010/get/&quot;)</span></span><br><span class="line">		<span class="comment"># proxy = random.choice(PROXIES)print(proxy,&quot;*&quot;*100)</span></span><br><span class="line">		<span class="comment"># if proxy[&#x27;user_pass&#x27;] is not None:</span></span><br><span class="line">		<span class="comment"># 	request.meta[&#x27;proxy&#x27;] = &quot;http://%s&quot; % proxy[&#x27;ip_port&#x27;]</span></span><br><span class="line">		<span class="comment"># 	encoded_user_pass = base64.encodestring(proxy[&#x27;user_pass&#x27;])</span></span><br><span class="line">		<span class="comment"># 	request.headers[&#x27;Proxy-Authorization&#x27;] = &#x27;Basic &#x27; + encoded_user_pass</span></span><br><span class="line">		<span class="comment"># 	print (&quot;**************ProxyMiddleware have pass************&quot;) + proxy[&#x27;ip_port&#x27;]</span></span><br><span class="line">		<span class="comment"># else:</span></span><br><span class="line">		<span class="comment"># 	# print (&quot;**************ProxyMiddleware no pass************&quot;) + proxy[&#x27;ip_port&#x27;]</span></span><br><span class="line">		<span class="comment"># 	request.meta[&#x27;proxy&#x27;] = &quot;http://%s&quot; % proxy[&#x27;ip_port&#x27;]</span></span><br><span class="line">        </span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> signals</span><br><span class="line"><span class="keyword">from</span> w3lib.http <span class="keyword">import</span> basic_auth_header</span><br><span class="line"><span class="keyword">import</span> base64</span><br><span class="line"><span class="keyword">from</span> scrapy.downloadermiddlewares.retry <span class="keyword">import</span> RetryMiddleware</span><br><span class="line"></span><br><span class="line"><span class="comment"># 代理服务器</span></span><br><span class="line">proxyServer = <span class="string">&quot;http://http-dyn.abuyun.com:9020&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 代理隧道验证信息</span></span><br><span class="line">proxyUser = <span class="string">&quot;H6587BH09900664D&quot;</span></span><br><span class="line">proxyPass = <span class="string">&quot;20C4314AF6C62E0F&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># for Python2</span></span><br><span class="line"><span class="comment"># proxyAuth = &quot;Basic &quot; + base64.b64encode(proxyUser + &quot;:&quot; + proxyPass)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># for Python3</span></span><br><span class="line">proxyAuth = <span class="string">&quot;Basic &quot;</span> + base64.urlsafe_b64encode(<span class="built_in">bytes</span>((proxyUser + <span class="string">&quot;:&quot;</span> + proxyPass), <span class="string">&quot;ascii&quot;</span>)).decode(<span class="string">&quot;utf8&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">RandomProxy</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_request</span>(<span class="params">self, request, spider</span>):</span></span><br><span class="line">        request.meta[<span class="string">&quot;proxy&quot;</span>] = proxyServer</span><br><span class="line">        self.ua =  redis_ua.sget(<span class="string">&quot;user-agent&quot;</span>)</span><br><span class="line">        request.headers[<span class="string">&quot;User-Agent&quot;</span>]=<span class="built_in">str</span>(self.ua,encoding=<span class="string">&quot;utf-8&quot;</span>)</span><br><span class="line">        request.headers[<span class="string">&quot;Proxy-Authorization&quot;</span>] = proxyAuth  </span><br><span class="line">        <span class="comment"># print(request)</span></span><br><span class="line">        request.cookies=&#123;</span><br><span class="line">            <span class="string">&quot;session-id&quot;</span>:<span class="string">&quot;134-0075797-0381056&quot;</span>,</span><br><span class="line">            <span class="string">&quot;session-id-time&quot;</span>:<span class="string">&quot;2082787201l&quot;</span>,</span><br><span class="line">            <span class="string">&quot;i18n-prefs&quot;</span>:<span class="string">&quot;USD&quot;</span>,</span><br><span class="line">            <span class="string">&quot;lc-main&quot;</span>:<span class="string">&quot;en_US&quot;</span>,</span><br><span class="line">            <span class="string">&quot;sp-cdn&quot;</span>:<span class="string">&quot;L5Z9:CN&quot;</span>,</span><br><span class="line">            <span class="string">&quot;ubid-main&quot;</span>:<span class="string">&quot;134-0448665-3732200&quot;</span>,</span><br><span class="line">            <span class="string">&quot;session-token&quot;</span>:<span class="string">&quot;KBeUZ2ocx7ObasfCr4XpPQE7t5RPj80VGzKb+EbFEEAMWbzwl8YuNtcPmqhZ6kUUtVxpwwOJSOy0XvZvPIKuxoUIgKbIuo6Z3qaX1Lk/0ewvaJY55aR+ayz/kJ9jUmL2HhLMwE8ipxIPqc/lzRRpAUxD4DuCd2PidfXY+sG5IhTVooAuK4UMzq1gB3Azo/BO&quot;</span></span><br><span class="line">        &#125;  </span><br><span class="line">    <span class="comment"># def process_request(self, request, spider):</span></span><br><span class="line">        <span class="comment"># proxy = &quot;tps121.kdlapi.com:15818&quot;</span></span><br><span class="line">        <span class="comment"># request.meta[&#x27;proxy&#x27;] = &quot;https://%s&quot; %proxy</span></span><br><span class="line">        <span class="comment"># # 用户名密码认证</span></span><br><span class="line">        <span class="comment"># request.headers[&#x27;Proxy-Authorization&#x27;] = basic_auth_header(&#x27;t19351016044852&#x27;, &#x27;y4asuamy&#x27;)  # 白名单认证可注释此行</span></span><br><span class="line">        <span class="comment"># return None</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># def __init__(self):</span></span><br><span class="line">    <span class="comment">#     self.proxy =  redis_ua.sget(&quot;daxiangIP&quot;)</span></span><br><span class="line">        </span><br><span class="line">    <span class="comment"># def process_request(self,request, spider):</span></span><br><span class="line">        <span class="comment"># proxy = requests.get(&quot;http://172.20.0.252:5010/get/&quot;).content</span></span><br><span class="line">        <span class="comment"># proxy=eval(str(proxy,encoding=&quot;utf-8&quot;))[&quot;proxy&quot;]</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># print(&quot;ip_proxy=&#123;&#125;&quot;.format(str(self.proxy,encoding=&quot;utf8&quot;)),&quot;*&quot;*30)</span></span><br><span class="line">        <span class="comment"># request.meta[&#x27;proxy&#x27;] = &#x27;https://27.188.65.244:8060&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># from fake_useragent import UserAgent</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">RandomUserAgentMiddlware</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line">    <span class="comment">#随机更换user-agent</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span>(<span class="params">self,crawler</span>):</span></span><br><span class="line">        <span class="built_in">super</span>(RandomUserAgentMiddlware,self).__init__()</span><br><span class="line">        <span class="comment"># self.ua =  random.choice(USER_AGENTS)</span></span><br><span class="line">        self.ua =  redis_ua.sget(<span class="string">&quot;user-agent&quot;</span>)</span><br><span class="line">        <span class="comment"># redis_ua.rpush(&#x27;user-agent&#x27;,self.ua)</span></span><br><span class="line">        self.ua=<span class="built_in">str</span>(self.ua,encoding=<span class="string">&#x27;utf-8&#x27;</span>)</span><br><span class="line">        <span class="comment"># self.ua =  UserAgent(verify_ssl=False)</span></span><br><span class="line"></span><br><span class="line"><span class="meta">    @classmethod</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span>(<span class="params">cls,crawler</span>):</span></span><br><span class="line">        <span class="keyword">return</span> cls(crawler)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_request</span>(<span class="params">self,request,spider</span>):</span></span><br><span class="line">        request.headers.setdefault(<span class="string">&quot;User-Agent&quot;</span>,self.ua)</span><br><span class="line">        <span class="comment"># request.headers.setdefault(&quot;User-Agent&quot;,self.ua.random)</span></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">GetFailedUrl</span>(<span class="params">RetryMiddleware</span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span>(<span class="params">self, settings</span>):</span></span><br><span class="line">        self.max_retry_times = settings.getint(<span class="string">&#x27;RETRY_TIMES&#x27;</span>)</span><br><span class="line">        self.retry_http_codes = <span class="built_in">set</span>(<span class="built_in">int</span>(x) <span class="keyword">for</span> x <span class="keyword">in</span> settings.getlist(<span class="string">&#x27;RETRY_HTTP_CODES&#x27;</span>))</span><br><span class="line">        self.priority_adjust = settings.getint(<span class="string">&#x27;RETRY_PRIORITY_ADJUST&#x27;</span>)</span><br><span class="line"> </span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_response</span>(<span class="params">self, request, response, spider</span>):</span></span><br><span class="line">        <span class="comment"># print(response.text)</span></span><br><span class="line">        <span class="keyword">if</span> response.status != <span class="number">200</span> <span class="keyword">or</span> <span class="string">R&quot;robot check&quot;</span> <span class="keyword">in</span> response.text:</span><br><span class="line">            <span class="keyword">with</span> <span class="built_in">open</span>(<span class="built_in">str</span>(spider.name) + <span class="string">&quot;.txt&quot;</span>, <span class="string">&quot;a&quot;</span>) <span class="keyword">as</span> f:</span><br><span class="line">                f.write(response.url + <span class="string">&quot;\n&quot;</span>)</span><br><span class="line">            <span class="comment"># print(response.url,&quot;*&quot;*100)</span></span><br><span class="line">            <span class="keyword">if</span> <span class="string">R&quot;www.amazon.com/s?k=&quot;</span> <span class="keyword">in</span> request_url:</span><br><span class="line">                db = MongoDB(<span class="string">&quot;AddTask&quot;</span>,<span class="string">&quot;Add_List&quot;</span>)</span><br><span class="line">                items=&#123;&#125;</span><br><span class="line">                items[<span class="string">&quot;cid&quot;</span>]=GetAmazonUrlCID(request_url)</span><br><span class="line">                items[<span class="string">&#x27;kwURL&#x27;</span>]=[]</span><br><span class="line">                items[<span class="string">&#x27;Finished_time&#x27;</span>]=parse(<span class="built_in">str</span>(datetime.now())[<span class="number">0</span>:<span class="number">19</span>])</span><br><span class="line">                items[<span class="string">&#x27;link_url&#x27;</span>]=request_url</span><br><span class="line">                items[<span class="string">&#x27;status&#x27;</span>]=<span class="string">&quot;failed&quot;</span></span><br><span class="line">                items[<span class="string">&#x27;error&#x27;</span>]=<span class="built_in">str</span>(exception)</span><br><span class="line">                <span class="keyword">if</span> db.getInfo(&#123;<span class="string">&#x27;id&#x27;</span>:<span class="built_in">int</span>(items[<span class="string">&quot;cid&quot;</span>])&#125;):</span><br><span class="line">                    db.updateDict(&#123;<span class="string">&#x27;id&#x27;</span>:<span class="built_in">int</span>(items[<span class="string">&#x27;cid&#x27;</span>])&#125;,&#123;<span class="string">&#x27;$set&#x27;</span>:items&#125;)</span><br><span class="line">                <span class="comment"># print(&quot;*&quot;*100,str(exception))</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                db_ = MongoDB(<span class="string">&quot;AmazonDB&quot;</span>,<span class="string">&quot;AmazonContent&quot;</span>)</span><br><span class="line">                items=&#123;&#125;</span><br><span class="line">                items[<span class="string">&#x27;id&#x27;</span>]=db_.getID(<span class="string">&quot;amazonUrl&quot;</span>)</span><br><span class="line">                items[<span class="string">&quot;cid&quot;</span>]=GetAmazonUrlCID(request_url)</span><br><span class="line">                items[<span class="string">&#x27;Finished_time&#x27;</span>]=parse(<span class="built_in">str</span>(datetime.now())[<span class="number">0</span>:<span class="number">19</span>])</span><br><span class="line">                items[<span class="string">&#x27;link_url&#x27;</span>]=request_url</span><br><span class="line">                items[<span class="string">&#x27;status&#x27;</span>]=<span class="string">&quot;failed&quot;</span></span><br><span class="line">                items[<span class="string">&#x27;error&#x27;</span>]=<span class="built_in">str</span>(exception)</span><br><span class="line">                db_.insertDict(items)</span><br><span class="line">            <span class="keyword">return</span> request</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="keyword">return</span> response</span><br><span class="line"> </span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_exception</span>(<span class="params">self, request, exception, spider</span>):</span></span><br><span class="line">        <span class="keyword">if</span> <span class="built_in">isinstance</span>(exception, self.EXCEPTIONS_TO_RETRY):</span><br><span class="line">            request_url=<span class="built_in">str</span>(request)[<span class="number">5</span>:-<span class="number">1</span>]</span><br><span class="line">            <span class="keyword">with</span> <span class="built_in">open</span>(<span class="built_in">str</span>(spider.name) + <span class="string">&quot;.txt&quot;</span>, <span class="string">&quot;a&quot;</span>) <span class="keyword">as</span> f:</span><br><span class="line">                f.write(request_url + <span class="string">&quot;\n&quot;</span>)</span><br><span class="line">                print(<span class="string">&quot;*&quot;</span>*<span class="number">20</span>,request_url)</span><br><span class="line">            <span class="keyword">if</span> <span class="string">R&quot;www.amazon.com/s?k=&quot;</span> <span class="keyword">in</span> request_url:</span><br><span class="line">                db = MongoDB(<span class="string">&quot;AddTask&quot;</span>,<span class="string">&quot;Add_List&quot;</span>)</span><br><span class="line">                items=&#123;&#125;</span><br><span class="line">                items[<span class="string">&quot;cid&quot;</span>]=GetAmazonUrlCID(request_url)</span><br><span class="line">                items[<span class="string">&#x27;kwURL&#x27;</span>]=[]</span><br><span class="line">                items[<span class="string">&#x27;Finished_time&#x27;</span>]=parse(<span class="built_in">str</span>(datetime.now())[<span class="number">0</span>:<span class="number">19</span>])</span><br><span class="line">                items[<span class="string">&#x27;link_url&#x27;</span>]=request_url</span><br><span class="line">                items[<span class="string">&#x27;status&#x27;</span>]=<span class="string">&quot;failed&quot;</span></span><br><span class="line">                items[<span class="string">&#x27;error&#x27;</span>]=<span class="built_in">str</span>(exception)</span><br><span class="line">                <span class="keyword">if</span> db.getInfo(&#123;<span class="string">&#x27;id&#x27;</span>:<span class="built_in">int</span>(items[<span class="string">&quot;cid&quot;</span>])&#125;):</span><br><span class="line">                    db.updateDict(&#123;<span class="string">&#x27;id&#x27;</span>:<span class="built_in">int</span>(items[<span class="string">&#x27;cid&#x27;</span>])&#125;,&#123;<span class="string">&#x27;$set&#x27;</span>:items&#125;)</span><br><span class="line">                <span class="comment"># print(&quot;*&quot;*100,str(exception))</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                db_ = MongoDB(<span class="string">&quot;AmazonDB&quot;</span>,<span class="string">&quot;AmazonContent&quot;</span>)</span><br><span class="line">                items=&#123;&#125;</span><br><span class="line">                items[<span class="string">&#x27;id&#x27;</span>]=db_.getID(<span class="string">&quot;amazonUrl&quot;</span>)</span><br><span class="line">                items[<span class="string">&quot;cid&quot;</span>]=GetAmazonUrlCID(request_url)</span><br><span class="line">                items[<span class="string">&#x27;Finished_time&#x27;</span>]=parse(<span class="built_in">str</span>(datetime.now())[<span class="number">0</span>:<span class="number">19</span>])</span><br><span class="line">                items[<span class="string">&#x27;link_url&#x27;</span>]=request_url</span><br><span class="line">                items[<span class="string">&#x27;status&#x27;</span>]=<span class="string">&quot;failed&quot;</span></span><br><span class="line">                items[<span class="string">&#x27;error&#x27;</span>]=<span class="built_in">str</span>(exception)</span><br><span class="line">                db_.insertDict(items)</span><br><span class="line">            <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># from scrapy import signals</span></span><br><span class="line"><span class="comment"># import pyppeteer</span></span><br><span class="line"><span class="comment"># import asyncio</span></span><br><span class="line"><span class="comment"># import os</span></span><br><span class="line"><span class="comment"># import json</span></span><br><span class="line"><span class="comment"># import redis</span></span><br><span class="line"><span class="comment"># from scrapy.http import HtmlResponse</span></span><br><span class="line"><span class="comment"># from Amazon.ConfigDB import RedisDB,RedisPool,MongoDB</span></span><br><span class="line"><span class="comment"># import logging</span></span><br><span class="line"><span class="comment"># pyppeteer_level = logging.WARNING</span></span><br><span class="line"><span class="comment"># logging.getLogger(&#x27;pyppeteer&#x27;).setLevel(pyppeteer_level)</span></span><br><span class="line"><span class="comment"># logging.getLogger(&#x27;websockets.protocol&#x27;).setLevel(pyppeteer_level)</span></span><br><span class="line"><span class="comment"># pyppeteer_logger = logging.getLogger(&#x27;pyppeteer&#x27;)</span></span><br><span class="line"><span class="comment"># pyppeteer_logger.setLevel(logging.WARNING)</span></span><br><span class="line"><span class="comment"># # redisconn=RedisDB(db=0)</span></span><br><span class="line"><span class="comment"># redis_ua = RedisPool()</span></span><br><span class="line"><span class="comment"># pyppeteer.DEBUG = False</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># def _patch_pyppeteer():</span></span><br><span class="line"><span class="comment">#     from typing import Any</span></span><br><span class="line"><span class="comment">#     from pyppeteer import connection, launcher</span></span><br><span class="line"><span class="comment">#     import websockets.client</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     class PatchedConnection(connection.Connection):  # type: ignore</span></span><br><span class="line"><span class="comment">#         def __init__(self, *args: Any, **kwargs: Any) -&gt; None:</span></span><br><span class="line"><span class="comment">#             super().__init__(*args, **kwargs)</span></span><br><span class="line"><span class="comment">#             # the _ws argument is not yet connected, can simply be replaced with another</span></span><br><span class="line"><span class="comment">#             # with better defaults.</span></span><br><span class="line"><span class="comment">#             self._ws = websockets.client.connect(</span></span><br><span class="line"><span class="comment">#                 self._url,</span></span><br><span class="line"><span class="comment">#                 loop=self._loop,</span></span><br><span class="line"><span class="comment">#                 # the following parameters are all passed to WebSocketCommonProtocol</span></span><br><span class="line"><span class="comment">#                 # which markes all three as Optional, but connect() doesn&#x27;t, hence the liberal</span></span><br><span class="line"><span class="comment">#                 # use of type: ignore on these lines.</span></span><br><span class="line"><span class="comment">#                 # fixed upstream but not yet released, see aaugustin/websockets#93ad88</span></span><br><span class="line"><span class="comment">#                 max_size=None,  # type: ignore</span></span><br><span class="line"><span class="comment">#                 ping_interval=None,  # type: ignore</span></span><br><span class="line"><span class="comment">#                 ping_timeout=None,  # type: ignore</span></span><br><span class="line"><span class="comment">#             )</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     connection.Connection = PatchedConnection</span></span><br><span class="line"><span class="comment">#     # also imported as a  global in pyppeteer.launcher</span></span><br><span class="line"><span class="comment">#     launcher.Connection = PatchedConnection</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># class DownloaderMiddleware(object):</span></span><br><span class="line"><span class="comment">#     # Not all methods need to be defined. If a method is not defined,</span></span><br><span class="line"><span class="comment">#     # scrapy acts as if the downloader middleware does not modify the</span></span><br><span class="line"><span class="comment">#     # passed objects.</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     def __init__(self):</span></span><br><span class="line"><span class="comment">#         # print(&quot;Init downloaderMiddleware use pypputeer.&quot;)</span></span><br><span class="line"><span class="comment">#         # os.environ[&#x27;PYPPETEER_CHROMIUM_REVISION&#x27;] = &#x27;588429&#x27;</span></span><br><span class="line"><span class="comment">#         # pyppeteer.DEBUG = False</span></span><br><span class="line"><span class="comment">#         print(os.environ.get(&#x27;PYPPETEER_CHROMIUM_REVISION&#x27;))</span></span><br><span class="line"><span class="comment">#         loop = asyncio.get_event_loop()</span></span><br><span class="line"><span class="comment">#         task = asyncio.ensure_future(self.getbrowser())</span></span><br><span class="line"><span class="comment">#         loop.run_until_complete(task)</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#         # self.browser = task.result()</span></span><br><span class="line"><span class="comment">#         # print(self.browser)</span></span><br><span class="line"><span class="comment">#         # print(self.page)</span></span><br><span class="line"><span class="comment">#         # self.page = await browser.newPage()</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     async def getbrowser(self):</span></span><br><span class="line"><span class="comment">#         ua=redis_ua.sget(&quot;user-agent&quot;)</span></span><br><span class="line"><span class="comment">#         redis_ua.rpush(&#x27;user-agent&#x27;,ua)</span></span><br><span class="line"><span class="comment">#         # proxies=redis_ua.sget(&quot;daxiangIP&quot;)</span></span><br><span class="line"><span class="comment">#         # redisconn.lpush(&#x27;proxies:ipx&#x27;,proxies)</span></span><br><span class="line"><span class="comment">#         # Proxies=str(proxies,encoding=&#x27;utf-8&#x27;)</span></span><br><span class="line"><span class="comment">#         # print(ua,Proxies,&quot;&gt;&quot;*30)</span></span><br><span class="line"><span class="comment">#         # self.browser = await pyppeteer.launch(&#123;&#x27;headless&#x27;: False,&#x27;timeout&#x27;:0, </span></span><br><span class="line"><span class="comment">#         #                                   &#x27;args&#x27;: [</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--window-size=&#123;1300&#125;,&#123;600&#125;&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--disable-extensions&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--hide-scrollbars&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--disable-bundled-ppapi-flash&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--mute-audio&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--no-sandbox&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--disable-setuid-sandbox&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--disable-gpu&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--disable-infobars&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--proxy-server=http://http-dyn.abuyun.com:9020&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                   ],</span></span><br><span class="line"><span class="comment">#         #                                   &#x27;dumpio&#x27;: True</span></span><br><span class="line"><span class="comment">#         #                                   &#125;)</span></span><br><span class="line"><span class="comment">#         self.browser = await pyppeteer.connect(&#123;&#x27;browserWSEndpoint&#x27;: &#x27;ws://172.20.3.221:3000?--proxy-server=http://http-dyn.abuyun.com:9020&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #     &#x27;headless&#x27;: False,&#x27;timeout&#x27;:0,</span></span><br><span class="line"><span class="comment">#         # #                                 #     &#x27;args&#x27;: [</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--window-size=&#123;1300&#125;,&#123;600&#125;&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--disable-extensions&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--hide-scrollbars&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--disable-bundled-ppapi-flash&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--mute-audio&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--no-sandbox&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--disable-setuid-sandbox&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--disable-gpu&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--disable-infobars&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--proxy-server=http://http-dyn.abuyun.com:9020&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #   ],</span></span><br><span class="line"><span class="comment">#         # #                                 #   &#x27;dumpio&#x27;: True</span></span><br><span class="line"><span class="comment">#                                             &#125;)</span></span><br><span class="line"><span class="comment">#         self.page = await self.browser.newPage()</span></span><br><span class="line"><span class="comment">#         # await page.setExtraHTTPHeaders(&#123;&#x27;Proxy-Authorization&#x27;: &#x27;Basic H6587BH09900664D:20C4314AF6C62E0F&#x27;&#125;)</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#         await self.page.authenticate(&#123;&quot;username&quot;: &quot;H6587BH09900664D&quot;, &quot;password&quot;: &quot;20C4314AF6C62E0F&quot;&#125;)          </span></span><br><span class="line"><span class="comment">#         # await self.page.setExtraHTTPHeaders(&quot;http://http-dyn.abuyun.com:9020&quot;) </span></span><br><span class="line"><span class="comment">#         # await self.page.setCookie(cookie)</span></span><br><span class="line"><span class="comment">#         await self.page.setViewport(viewport=&#123;&#x27;width&#x27;: 1366, &#x27;height&#x27;: 768&#125;)</span></span><br><span class="line"><span class="comment">#         await self.page.setUserAgent(str(ua,encoding=&#x27;utf-8&#x27;))</span></span><br><span class="line"><span class="comment">#         # await self.page.setUserAgent(&quot;Chrome (AppleWebKit/537.1; Chrome50.0; Windows NT 6.3) AppleWebKit/537.36 (KHTML like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393&quot;)</span></span><br><span class="line"><span class="comment">#         await self.page.setJavaScriptEnabled(enabled=True)</span></span><br><span class="line"><span class="comment">#         await self.page.evaluate(</span></span><br><span class="line"><span class="comment">#             &#x27;&#x27;&#x27;() =&gt;&#123; Object.defineProperties(navigator,&#123; webdriver:&#123; get: () =&gt; false &#125; &#125;) &#125;&#x27;&#x27;&#x27;)  </span></span><br><span class="line"><span class="comment">#         await self.page.evaluate(&#x27;&#x27;&#x27;() =&gt;&#123; window.navigator.chrome = &#123; runtime: &#123;&#125;,  &#125;; &#125;&#x27;&#x27;&#x27;)</span></span><br><span class="line"><span class="comment">#         await self.page.evaluate(</span></span><br><span class="line"><span class="comment">#             &#x27;&#x27;&#x27;() =&gt;&#123; Object.defineProperty(navigator, &#x27;languages&#x27;, &#123; get: () =&gt; [&#x27;en-US&#x27;, &#x27;en&#x27;] &#125;); &#125;&#x27;&#x27;&#x27;)</span></span><br><span class="line"><span class="comment">#         await self.page.evaluate(</span></span><br><span class="line"><span class="comment">#             &#x27;&#x27;&#x27;() =&gt;&#123; Object.defineProperty(navigator, &#x27;plugins&#x27;, &#123; get: () =&gt; [1, 2, 3, 4, 5,6], &#125;); &#125;&#x27;&#x27;&#x27;)</span></span><br><span class="line"><span class="comment">#         await self.page.waitFor(10000)</span></span><br><span class="line"><span class="comment">#         # cooki=redisconn.get(&quot;ali1688cookie&quot;)</span></span><br><span class="line"><span class="comment">#         # redisconn.rpush(&#x27;ali1688cookie&#x27;,cooki)</span></span><br><span class="line"><span class="comment">#         # cookieStr=str(cooki,encoding=&#x27;utf-8&#x27;)</span></span><br><span class="line"><span class="comment">#         # for cook in json.loads(cookieStr):</span></span><br><span class="line"><span class="comment">#         #     await self.page.setCookie(cook)</span></span><br><span class="line"><span class="comment">#         return self.page</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">#     @classmethod</span></span><br><span class="line"><span class="comment">#     def from_crawler(cls, crawler):</span></span><br><span class="line"><span class="comment">#         # This method is used by Scrapy to create your spiders.</span></span><br><span class="line"><span class="comment">#         s = cls()</span></span><br><span class="line"><span class="comment">#         crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)</span></span><br><span class="line"><span class="comment">#         return s</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     def process_request(self, request, spider):</span></span><br><span class="line"><span class="comment">#         # Called for each request that goes through the downloader</span></span><br><span class="line"><span class="comment">#         # middleware.</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#         # Must either:</span></span><br><span class="line"><span class="comment">#         # - return None: continue processing this request</span></span><br><span class="line"><span class="comment">#         # - or return a Response object</span></span><br><span class="line"><span class="comment">#         # - or return a Request object</span></span><br><span class="line"><span class="comment">#         # - or raise IgnoreRequest: process_exception() methods of</span></span><br><span class="line"><span class="comment">#         #   installed downloader middleware will be called</span></span><br><span class="line"><span class="comment">#         loop = asyncio.get_event_loop()</span></span><br><span class="line"><span class="comment">#         task = asyncio.ensure_future(self.usePypuppeteer(request))</span></span><br><span class="line"><span class="comment">#         loop.run_until_complete(task)</span></span><br><span class="line"><span class="comment">#         # return task.result()</span></span><br><span class="line"><span class="comment">#         return HtmlResponse(url=request.url, body=task.result(), encoding=&quot;utf-8&quot;, request=request)</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     async def intercept_request(self,req):</span></span><br><span class="line"><span class="comment">#         if req.resourceType in [&#x27;image&#x27;, &#x27;media&#x27;, &#x27;eventsource&#x27;, &#x27;websocket&#x27;]:</span></span><br><span class="line"><span class="comment">#             await req.abort()</span></span><br><span class="line"><span class="comment">#         else:</span></span><br><span class="line"><span class="comment">#             await req.continue_()</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     async def intercept_response(self,res):</span></span><br><span class="line"><span class="comment">#         resourceType = res.request.resourceType</span></span><br><span class="line"><span class="comment">#         # if resourceType in [&#x27;xhr&#x27;, &#x27;fetch&#x27;]:</span></span><br><span class="line"><span class="comment">#         #     resp = await res.text()</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">#     async def usePypuppeteer(self, request):</span></span><br><span class="line"><span class="comment">#         ws = self.browser.wsEndpoint</span></span><br><span class="line"><span class="comment">#         # print(ws,&quot;running...&quot;)</span></span><br><span class="line"><span class="comment">#         while True:</span></span><br><span class="line"><span class="comment">#             try:</span></span><br><span class="line"><span class="comment">#                 await self.page.goto(request.url,&#123;&#x27;timeout&#x27;:0,&#x27;waitUntil&#x27;: &#x27;domcontentloaded&#x27;&#125;)</span></span><br><span class="line"><span class="comment">#                 if R&#x27;/s?k=&#x27; in request.url :</span></span><br><span class="line"><span class="comment">#                     next_url = await self.page.querySelector(&#x27;.a-last&#x27;)</span></span><br><span class="line"><span class="comment">#                 else:</span></span><br><span class="line"><span class="comment">#                     next_url = await self.page.querySelector(&#x27;#title&#x27;)</span></span><br><span class="line"><span class="comment">#                 print(next_url)</span></span><br><span class="line"><span class="comment">#                 if not next_url:</span></span><br><span class="line"><span class="comment">#                     continue</span></span><br><span class="line"><span class="comment">#                 break</span></span><br><span class="line"><span class="comment">#             except Exception as ex:</span></span><br><span class="line"><span class="comment">#                 print(&quot;network error ...&gt;&gt;&gt;1&quot;)</span></span><br><span class="line"><span class="comment">#                 # await self.page.close()</span></span><br><span class="line"><span class="comment">#                 await self.usePypuppeteer(request)</span></span><br><span class="line"><span class="comment">#                 # self.page.close()</span></span><br><span class="line"><span class="comment">#                 # await self.browser.disconnect()  </span></span><br><span class="line"><span class="comment">#                 # self.browser = await pyppeteer.connect(&#123;&#x27;browserWSEndpoint&#x27;: ws&#125;)</span></span><br><span class="line"><span class="comment">#                 # ua=redis_ua.sget(&quot;user-agent&quot;)</span></span><br><span class="line"><span class="comment">#                 # redis_ua.rpush(&#x27;user-agent&#x27;,ua)</span></span><br><span class="line"><span class="comment">#                 # self.page = await self.browser.newPage()</span></span><br><span class="line"><span class="comment">#                 # await self.page.authenticate(&#123;&quot;username&quot;: &quot;H6587BH09900664D&quot;, &quot;password&quot;: &quot;20C4314AF6C62E0F&quot;&#125;)          </span></span><br><span class="line"><span class="comment">#                 # await self.page.setViewport(viewport=&#123;&#x27;width&#x27;: 1366, &#x27;height&#x27;: 768&#125;)</span></span><br><span class="line"><span class="comment">#                 # await self.page.setUserAgent(str(ua,encoding=&#x27;utf-8&#x27;))</span></span><br><span class="line"><span class="comment">#                 # await self.page.setJavaScriptEnabled(enabled=True)</span></span><br><span class="line"><span class="comment">#                 # await self.page.evaluate(</span></span><br><span class="line"><span class="comment">#                 #     &#x27;&#x27;&#x27;() =&gt;&#123; Object.defineProperties(navigator,&#123; webdriver:&#123; get: () =&gt; false &#125; &#125;) &#125;&#x27;&#x27;&#x27;)  </span></span><br><span class="line"><span class="comment">#                 # await self.page.evaluate(&#x27;&#x27;&#x27;() =&gt;&#123; window.navigator.chrome = &#123; runtime: &#123;&#125;,  &#125;; &#125;&#x27;&#x27;&#x27;)</span></span><br><span class="line"><span class="comment">#                 # await self.page.evaluate(</span></span><br><span class="line"><span class="comment">#                 #     &#x27;&#x27;&#x27;() =&gt;&#123; Object.defineProperty(navigator, &#x27;languages&#x27;, &#123; get: () =&gt; [&#x27;en-US&#x27;, &#x27;en&#x27;] &#125;); &#125;&#x27;&#x27;&#x27;)</span></span><br><span class="line"><span class="comment">#                 # await self.page.evaluate(</span></span><br><span class="line"><span class="comment">#                 #     &#x27;&#x27;&#x27;() =&gt;&#123; Object.defineProperty(navigator, &#x27;plugins&#x27;, &#123; get: () =&gt; [1, 2, 3, 4, 5,6], &#125;); &#125;&#x27;&#x27;&#x27;)</span></span><br><span class="line"><span class="comment">#                 # await self.page.waitFor(10000)</span></span><br><span class="line"><span class="comment">#                 # self.saveMongo(request,ex)</span></span><br><span class="line"><span class="comment">#                 continue</span></span><br><span class="line"><span class="comment">#         if R&#x27;/s?k=&#x27; in request.url :</span></span><br><span class="line"><span class="comment">#             await self.page.setRequestInterception(True)</span></span><br><span class="line"><span class="comment">#             self.page.on(&#x27;request&#x27;, self.intercept_request)</span></span><br><span class="line"><span class="comment">#             # self.page.on(&#x27;response&#x27;, self.intercept_response)</span></span><br><span class="line"><span class="comment">#             await self.page.evaluate(&#x27;window.scrollBy(0, document.body.scrollHeight)&#x27;)</span></span><br><span class="line"><span class="comment">#             await asyncio.sleep(3)</span></span><br><span class="line"><span class="comment">#         else:</span></span><br><span class="line"><span class="comment">#             # await asyncio.sleep(3)</span></span><br><span class="line"><span class="comment">#             #鼠标滚动到底</span></span><br><span class="line"><span class="comment">#             for i in range(0,6):</span></span><br><span class="line"><span class="comment">#                 await self.page.evaluate(&#x27;window.scrollBy(0, &#123;&#125;)&#x27;.format(800*i))</span></span><br><span class="line"><span class="comment">#                 await asyncio.sleep(4)</span></span><br><span class="line"><span class="comment">#             # await self.page.evaluate(&#x27;window.scrollBy(0, document.body.scrollHeight*1/2)&#x27;)</span></span><br><span class="line"><span class="comment">#             # await asyncio.sleep(3)</span></span><br><span class="line"><span class="comment">#         content = await self.page.content()</span></span><br><span class="line"><span class="comment">#         # content = await self.page.evaluate(&#x27;document.body.textContent&#x27;, force_expr=True) </span></span><br><span class="line"><span class="comment">#         return content</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     def saveMongo(self, request, exception):</span></span><br><span class="line"><span class="comment">#         if exception:</span></span><br><span class="line"><span class="comment">#             print(request.url)</span></span><br><span class="line"><span class="comment">#             request_url=str(request.url)</span></span><br><span class="line"><span class="comment">#             if R&quot;www.amazon.com/s?k=&quot; in request_url:</span></span><br><span class="line"><span class="comment">#                 db = MongoDB(&quot;AddTask&quot;,&quot;Add_List&quot;)</span></span><br><span class="line"><span class="comment">#                 items=&#123;&#125;</span></span><br><span class="line"><span class="comment">#                 items[&quot;cid&quot;]=GetAmazonUrlCID(request_url)</span></span><br><span class="line"><span class="comment">#                 items[&#x27;kwURL&#x27;]=[]</span></span><br><span class="line"><span class="comment">#                 items[&#x27;Finished_time&#x27;]=parse(str(datetime.now())[0:19])</span></span><br><span class="line"><span class="comment">#                 items[&#x27;link_url&#x27;]=request_url</span></span><br><span class="line"><span class="comment">#                 items[&#x27;status&#x27;]=&quot;failed&quot;</span></span><br><span class="line"><span class="comment">#                 items[&#x27;error&#x27;]=str(exception)</span></span><br><span class="line"><span class="comment">#                 if db.getInfo(&#123;&#x27;id&#x27;:int(items[&quot;cid&quot;])&#125;):</span></span><br><span class="line"><span class="comment">#                     db.updateDict(&#123;&#x27;id&#x27;:int(items[&#x27;cid&#x27;])&#125;,&#123;&#x27;$set&#x27;:items&#125;)</span></span><br><span class="line"><span class="comment">#                 # print(&quot;*&quot;*100,str(exception))</span></span><br><span class="line"><span class="comment">#             else:</span></span><br><span class="line"><span class="comment">#                 db_ = MongoDB(&quot;AmazonDB&quot;,&quot;AmazonContent&quot;)</span></span><br><span class="line"><span class="comment">#                 items=&#123;&#125;</span></span><br><span class="line"><span class="comment">#                 items[&#x27;id&#x27;]=db_.getID(&quot;amazonUrl&quot;)</span></span><br><span class="line"><span class="comment">#                 items[&quot;cid&quot;]=GetAmazonUrlCID(request_url)</span></span><br><span class="line"><span class="comment">#                 items[&#x27;Finished_time&#x27;]=parse(str(datetime.now())[0:19])</span></span><br><span class="line"><span class="comment">#                 items[&#x27;link_url&#x27;]=request_url</span></span><br><span class="line"><span class="comment">#                 items[&#x27;status&#x27;]=&quot;failed&quot;</span></span><br><span class="line"><span class="comment">#                 items[&#x27;error&#x27;]=str(exception)</span></span><br><span class="line"><span class="comment">#                 db_.insertDict(items)</span></span><br><span class="line"><span class="comment">#             return None</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     def process_response(self, request, response, spider):</span></span><br><span class="line"><span class="comment">#         # Called with the response returned from the downloader.</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#         # Must either;</span></span><br><span class="line"><span class="comment">#         # - return a Response object</span></span><br><span class="line"><span class="comment">#         # - return a Request object</span></span><br><span class="line"><span class="comment">#         # - or raise IgnoreRequest</span></span><br><span class="line"><span class="comment">#         return response</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     def process_exception(self, request, exception, spider):</span></span><br><span class="line"><span class="comment">#         # Called when a download handler or a process_request()</span></span><br><span class="line"><span class="comment">#         # (from other downloader middleware) raises an exception.</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#         # Must either:</span></span><br><span class="line"><span class="comment">#         # - return None: continue processing this exception</span></span><br><span class="line"><span class="comment">#         # - return a Response object: stops process_exception() chain</span></span><br><span class="line"><span class="comment">#         # - return a Request object: stops process_exception() chain</span></span><br><span class="line"><span class="comment">#         pass</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     def spider_opened(self, spider):</span></span><br><span class="line"><span class="comment">#         spider.logger.info(&#x27;Spider opened: %s&#x27; % spider.name)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># _patch_pyppeteer()</span></span><br></pre></td></tr></table></figure>











    </div>

    
    
    
        <div class="reward-container">
  <div>Thanks.</div>
  <button onclick="var qr = document.getElementById('qr'); qr.style.display = (qr.style.display === 'none') ? 'block' : 'none';">
    Donate
  </button>
  <div id="qr" style="display: none;">
      
      <div style="display: inline-block;">
        <img src="/images/wechatpay.png" alt="Hory Skone WeChat Pay">
        <p>WeChat Pay</p>
      </div>

  </div>
</div>

        

<div>
<ul class="post-copyright">
  <li class="post-copyright-author">
    <strong>Post author:  </strong>Hory Skone
  </li>
  <li class="post-copyright-link">
    <strong>Post link: </strong>
    <a href="http://blog.hory-ai.com/2020/07/29/%E8%8E%B7%E5%8F%96scrapy-request-%E5%A4%B4%E9%83%A8%E4%BF%A1%E6%81%AF%E5%8F%8A%E6%8D%95%E6%8D%89%E5%BC%82%E5%B8%B8%E4%BF%A1%E6%81%AF/" title="获取scrapy request 头部信息及捕捉异常信息">http://blog.hory-ai.com/2020/07/29/获取scrapy-request-头部信息及捕捉异常信息/</a>
  </li>
  <li class="post-copyright-license">
    <strong>Copyright Notice:  </strong>All articles in this blog are licensed under <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="noopener" target="_blank"><i class="fab fa-fw fa-creative-commons"></i>BY-NC-SA</a> unless stating additionally.
  </li>
</ul>
</div>


      <footer class="post-footer">

        


        
    <div class="post-nav">
      <div class="post-nav-item">
    <a href="/2020/07/29/HyperLedger-Fabric-2-0-release%E6%B5%8B%E8%AF%95%E7%BD%91%E7%BB%9C%E9%83%A8%E7%BD%B2/" rel="prev" title="HyperLedger Fabric 2.0-release测试网络部署">
      <i class="fa fa-chevron-left"></i> HyperLedger Fabric 2.0-release测试网络部署
    </a></div>
      <div class="post-nav-item">
    <a href="/2020/07/30/%E5%8D%95%E6%9C%BA%E7%89%88%E2%80%94%E2%80%94docker%E6%90%AD%E5%BB%BAMongoDB%E5%88%86%E7%89%87%E9%9B%86%E7%BE%A4/" rel="next" title="单机版——docker搭建MongoDB分片集群">
      单机版——docker搭建MongoDB分片集群 <i class="fa fa-chevron-right"></i>
    </a></div>
    </div>
      </footer>
    
  </article>
  
  
  



          </div>
          
    <div class="comments" id="valine-comments"></div>

<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      let commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          Table of Contents
        </li>
        <li class="sidebar-nav-overview">
          Overview
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#scrapy-middleware%E4%B8%AD%E9%97%B4%E4%BB%B6"><span class="nav-number">1.</span> <span class="nav-text">scrapy  middleware中间件</span></a></li></ol></div>
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">Hory Skone</p>
  <div class="site-description" itemprop="description"></div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">67</span>
          <span class="site-state-item-name">posts</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">30</span>
        <span class="site-state-item-name">categories</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">35</span>
        <span class="site-state-item-name">tags</span></a>
      </div>
  </nav>
</div>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
        <a href="https://github.com/horysk" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;horysk" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
      </span>
      <span class="links-of-author-item">
        <a href="mailto:admin@horysk.com" title="E-Mail → mailto:admin@horysk.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>E-Mail</a>
      </span>
      <span class="links-of-author-item">
        <a href="http://www.hory-ai.com/" title="HoryAI → http:&#x2F;&#x2F;www.hory-ai.com" rel="noopener" target="_blank"><i class="fab fa-google fa-fw"></i>HoryAI</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://blog.csdn.net/sirobot" title="CSDN → https:&#x2F;&#x2F;blog.csdn.net&#x2F;sirobot" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>CSDN</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://www.zhihu.com/people/AI_HH" title="ZhiHu → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;AI_HH" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>ZhiHu</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://www.kaggle.com/" title="Kaggle → https:&#x2F;&#x2F;www.kaggle.com&#x2F;" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>Kaggle</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://tianchi.aliyun.com/competition/gameList/activeList" title="TianChi → https:&#x2F;&#x2F;tianchi.aliyun.com&#x2F;competition&#x2F;gameList&#x2F;activeList" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>TianChi</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://works.yangerxiao.com/honeyed-words-generator" title="土情话 → https:&#x2F;&#x2F;works.yangerxiao.com&#x2F;honeyed-words-generator" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>土情话</a>
      </span>
  </div>



      </div>

      
      <script type="text/javascript" charset="utf-8" src="/js/tagcloud.js"></script>
      <script type="text/javascript" charset="utf-8" src="/js/tagcanvas.js"></script>
      <div class="widget-wrap">
          <h3 class="widget-title">Tag Cloud</h3>
          <div id="myCanvasContainer" class="widget tagcloud">
              <canvas width="250" height="250" id="resCanvas" style="width:100%">
                  <ul class="tag-list" itemprop="keywords"><li class="tag-list-item"><a class="tag-list-link" href="/tags/AI/" rel="tag">AI</a><span class="tag-list-count">2</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/BI/" rel="tag">BI</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Centos/" rel="tag">Centos</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/DNS/" rel="tag">DNS</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Dapp/" rel="tag">Dapp</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Docker/" rel="tag">Docker</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Hack/" rel="tag">Hack</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Hexo/" rel="tag">Hexo</a><span class="tag-list-count">6</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/HyperLedger-Fabric/" rel="tag">HyperLedger Fabric</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Hyperledger-Fabric/" rel="tag">Hyperledger Fabric</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Iftop/" rel="tag">Iftop</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Linux/" rel="tag">Linux</a><span class="tag-list-count">9</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/ML/" rel="tag">ML</a><span class="tag-list-count">5</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/MTProxy/" rel="tag">MTProxy</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Mongo/" rel="tag">Mongo</a><span class="tag-list-count">4</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Network/" rel="tag">Network</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Pyppeteer/" rel="tag">Pyppeteer</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Tools/" rel="tag">Tools</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/VPN/" rel="tag">VPN</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Vnc/" rel="tag">Vnc</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/baostock/" rel="tag">baostock</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/block-chain/" rel="tag">block chain</a><span class="tag-list-count">3</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/centos-xfce-vnc/" rel="tag">centos-xfce-vnc</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/docker/" rel="tag">docker</a><span class="tag-list-count">11</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/fabric/" rel="tag">fabric</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/hexo/" rel="tag">hexo</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/horysk/" rel="tag">horysk</a><span class="tag-list-count">3</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/k8s/" rel="tag">k8s</a><span class="tag-list-count">5</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/linux/" rel="tag">linux</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/mongo/" rel="tag">mongo</a><span class="tag-list-count">2</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/pyppeteer/" rel="tag">pyppeteer</a><span class="tag-list-count">2</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/python/" rel="tag">python</a><span class="tag-list-count">4</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/quant/" rel="tag">quant</a><span class="tag-list-count">7</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/stock/" rel="tag">stock</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/tushare/" rel="tag">tushare</a><span class="tag-list-count">1</span></li></ul>
              </canvas>
          </div>
      </div>
      
    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

        
  <div class="beian"><a href="http://www.beian.miit.gov.cn/" rel="noopener" target="_blank">null </a>
  </div>

<div class="copyright">
  
  &copy; 2015 – 
  <span itemprop="copyrightYear">2022</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Hory Skone</span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item-icon">
      <i class="fa fa-chart-area"></i>
    </span>
    <span title="Symbols count total">275k</span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item-icon">
      <i class="fa fa-coffee"></i>
    </span>
    <span title="Reading time total">4:10</span>
</div>

        
<div class="busuanzi-count">
  <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
    <span class="post-meta-item" id="busuanzi_container_site_uv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-user"></i>
      </span>
      <span class="site-uv" title="Total Visitors">
        <span id="busuanzi_value_site_uv"></span>
      </span>
    </span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item" id="busuanzi_container_site_pv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-eye"></i>
      </span>
      <span class="site-pv" title="Total Views">
        <span id="busuanzi_value_site_pv"></span>
      </span>
    </span>
</div>








      </div>
    </footer>
  </div>

  
  
  <script color='255,255,255' opacity='0.6' zIndex='-1' count='99' src="/lib/canvas-nest/canvas-nest.min.js"></script>
  <script size="300" alpha="0.6" zIndex="-1" src="/lib/canvas-ribbon/canvas-ribbon.js"></script>
  <script src="/lib/anime.min.js"></script>
  <script src="/lib/velocity/velocity.min.js"></script>
  <script src="/lib/velocity/velocity.ui.min.js"></script>

<script src="/js/utils.js"></script>

<script src="/js/motion.js"></script>


<script src="/js/schemes/pisces.js"></script>


<script src="/js/next-boot.js"></script>

<script src="/js/bookmark.js"></script>


  <script defer src="/lib/three/three.min.js"></script>
    <script defer src="/lib/three/three-waves.min.js"></script>
    <script defer src="/lib/three/canvas_lines.min.js"></script>
    <script defer src="/lib/three/canvas_sphere.min.js"></script>


  
  <script>
    (function(){
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x=document.getElementsByTagName("link");
		//Find the last canonical URL
		if(x.length > 0){
			for (i=0;i<x.length;i++){
				if(x[i].rel.toLowerCase() == 'canonical' && x[i].href){
					canonicalURL=x[i].href;
				}
			}
		}
    //Get protocol
	    if (!canonicalURL){
	    	curProtocol = window.location.protocol.split(':')[0];
	    }
	    else{
	    	curProtocol = canonicalURL.split(':')[0];
	    }
      //Get current URL if the canonical URL does not exist
	    if (!canonicalURL) canonicalURL = window.location.href;
	    //Assign script content. Replace current URL with the canonical URL
      !function(){var e=/([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,r=canonicalURL,t=document.referrer;if(!e.test(r)){var n=(String(curProtocol).toLowerCase() === 'https')?"https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif":"//api.share.baidu.com/s.gif";t?(n+="?r="+encodeURIComponent(document.referrer),r&&(n+="&l="+r)):r&&(n+="?l="+r);var i=new Image;i.src=n}}(window);})();
  </script>




  
<script src="/js/local-search.js"></script>













  

  


<script>
NexT.utils.loadComments(document.querySelector('#valine-comments'), () => {
  NexT.utils.getScript('https://cdn.jsdelivr.net/npm/valine@1.3.9/dist/Valine.min.js', () => {
    var GUEST = ['nick', 'mail', 'link'];
    var guest = 'nick,mail,link';
    guest = guest.split(',').filter(item => {
      return GUEST.includes(item);
    });
    new Valine({
      el         : '#valine-comments',
      verify     : false,
      notify     : false,
      appId      : '2ASH47v4VIRijd91HeEi3pws-gzGzoHsz',
      appKey     : 'ojRfR89cSV614pgJFT36lFFB',
      placeholder: "昵称填写qq可以显示qq头像和昵称哦~",
      avatar     : 'monsterid',
      meta       : guest,
      pageSize   : '10' || 10,
      visitor    : false,
      lang       : '' || 'zh-cn',
      path       : location.pathname,
      recordIP   : true,
      serverURLs : '',
      requiredFields: ['nick','mail'] 
    });
  }, window.Valine);
});
</script>

  <!-- 页面点击特效 -->
  <script type="text/javascript" src="/js/jquery-3.3.1.min.js"></script>
  <script type="text/javascript" src="/js/click-word.js"></script>
  
  <script type="text/javascript"
  color="255,255,255" opacity='0.7' zIndex="-2" count="500" src="//cdn.bootcss.com/canvas-nest.js/1.0.0/canvas-nest.min.js"></script>
  
<script src="/live2dw/lib/L2Dwidget.min.js?094cbace49a39548bed64abff5988b05"></script><script>L2Dwidget.init({"pluginRootPath":"live2dw/","pluginJsPath":"lib/","pluginModelPath":"assets/","tagMode":false,"log":false,"model":{"jsonPath":"/live2dw/assets/z16.model.json"},"display":{"position":"right","width":300,"height":600},"mobile":{"show":true}});</script></body>
</html>
